{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "import env\n",
    "from utils import (KaggleCameraDataset, one_hot_decision_function, \n",
    "                   unhot, softmax, inv_softmax, progress_iter, RNG, float32)\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## useful routines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_proba(proba_path):\n",
    "    return pd.read_csv(proba_path).as_matrix(columns=map(str, range(10)))\n",
    "\n",
    "def save_proba(P, path):\n",
    "    test_dataset = KaggleCameraDataset('../data/', train=False, lazy=True)\n",
    "    fnames = [os.path.split(fname)[-1] for fname in test_dataset.X]\n",
    "    df = pd.DataFrame(P)\n",
    "    df['fname'] = fnames\n",
    "    df = df[['fname'] + range(10)]\n",
    "    df.to_csv(os.path.join(path), index=False)\n",
    "\n",
    "def avg_proba(*proba_matrices, **kwargs):\n",
    "    #return np.exp(np.average([np.log(1e-16 + get_proba(path)) for path in proba_paths], axis=0, weights=kwargs.get('weights', None)))\n",
    "    P = softmax(np.average([inv_softmax(P) for P in proba_matrices], axis=0, weights=kwargs.get('weights', None)))\n",
    "#     P = np.average(proba_matrices, axis=0, weights=kwargs.get('weights', None))\n",
    "    return P\n",
    "\n",
    "\n",
    "def avg_proba_paths(*proba_paths, **kwargs):\n",
    "    return avg_proba(*map(get_proba, proba_paths), **kwargs)\n",
    "\n",
    "\n",
    "def proba_to_subm(proba, subm_path):\n",
    "    \"\"\"\n",
    "    Parameters\n",
    "    ----------\n",
    "    proba : (2640, 10) np.ndarray\n",
    "    \"\"\"\n",
    "    test_dataset = KaggleCameraDataset('../data/', train=False, lazy=True)\n",
    "    fnames = [os.path.split(fname)[-1] for fname in test_dataset.X]\n",
    "    index_pred = unhot(one_hot_decision_function(proba))\n",
    "    data = {'fname': fnames,\n",
    "            'camera': [KaggleCameraDataset.target_labels()[int(c)] for c in index_pred]}\n",
    "    df = pd.DataFrame(data, columns=['fname', 'camera'])\n",
    "    df.to_csv(subm_path, index=False)\n",
    "\n",
    "def subm(ind, subm_path):\n",
    "    \"\"\"\n",
    "    Parameters\n",
    "    ----------\n",
    "    proba : (2640, 10) np.ndarray\n",
    "    \"\"\"\n",
    "    test_dataset = KaggleCameraDataset('../data/', train=False, lazy=True)\n",
    "    fnames = [os.path.split(fname)[-1] for fname in test_dataset.X]\n",
    "    index_pred = ind\n",
    "    data = {'fname': fnames,\n",
    "            'camera': [KaggleCameraDataset.target_labels()[int(c)] for c in index_pred]}\n",
    "    df = pd.DataFrame(data, columns=['fname', 'camera'])\n",
    "    df.to_csv(subm_path, index=False)\n",
    "\n",
    "def get_diff(subm_path1, subm_path2):\n",
    "    df1 = pd.read_csv(subm_path1)\n",
    "    df2 = pd.read_csv(subm_path2)\n",
    "    return (df1 != df2)['camera'].sum()/2640."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  2.30427057e-01,   4.84325022e-01,   5.16182883e-03],\n",
       "       [  4.63270843e-02,   1.69466971e-03,   1.10656278e-04],\n",
       "       [  1.59370884e-01,   1.33133292e-01,   2.71781767e-03]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "P = get_proba('../models/proba1.csv')\n",
    "P[:3, :3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## #1 average different predictions (from raw probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  1.30430266e-01,   4.46921155e-01,   3.23994139e-02, ...,\n",
       "          1.20126526e-03,   1.05716349e-02,   6.87616952e-02],\n",
       "       [  3.24848127e-02,   8.78940028e-02,   4.32265330e-03, ...,\n",
       "          8.78389790e-04,   6.59955412e-01,   4.10077199e-02],\n",
       "       [  1.20631762e-01,   1.52186640e-01,   1.57862445e-03, ...,\n",
       "          1.04656043e-03,   1.19606800e-01,   5.12661040e-01],\n",
       "       ..., \n",
       "       [  1.16247602e-01,   4.08802994e-01,   5.17931608e-02, ...,\n",
       "          3.80601780e-02,   1.16218922e-02,   1.64976966e-01],\n",
       "       [  4.96196985e-01,   4.23220114e-03,   2.48291728e-03, ...,\n",
       "          3.88299840e-01,   1.79937498e-02,   6.91680338e-02],\n",
       "       [  2.15672876e-01,   3.66537021e-02,   2.02466367e-03, ...,\n",
       "          1.86312740e-05,   3.95222452e-02,   1.85410858e-02]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q = avg_proba('../models/proba1.csv', \n",
    "              '../models/proba_best.csv', \n",
    "               weights=[1., 1.])\n",
    "Q"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## #2 convert proba to predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "proba_to_subm(Q, 'subm.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## #3 how predictions differ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/pandas/core/computation/check.py:17: UserWarning: The installed version of numexpr 2.4.3 is not supported in pandas and will be not be used\n",
      "The minimum supported version is 2.4.6\n",
      "\n",
      "  ver=ver, min_ver=_MIN_NUMEXPR_VERSION), UserWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.4693181818181818"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_diff('../models/submission1.csv', '../models/submission_best.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ensembles, yesss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generations 1+2 (manual weights) $\\rightarrow$ 0.913"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "proba_dirs = [\n",
    "    #\n",
    "    ## 1th-generation\n",
    "    ### \n",
    "    # DenseNet no augmentation (512x512 crops)\n",
    "    (0.1, '0.579-#15'),\n",
    "    # DenseNet + random horiz flips only\n",
    "    (0.25, '0.691-#23-tta-horiz'),\n",
    "    # DenseNet + random crops + aug + TTA x 10 + LR restart\n",
    "    (0.25, '0.732-#23-tta10'),\n",
    "    (1., '0.750-#25'),\n",
    "    # ... but trained on 256x256 crops\n",
    "    (1., '0.742-#36'),\n",
    "    # ... but trained using Hinge Loss (512x512)\n",
    "    (1., '0.750-#30'),\n",
    "    # CNN_Small 1-FC no aug\n",
    "    (0.25, '0.665-#39'),\n",
    "    # CNN_Small 2-FC (using best TTA) + SGD-m\n",
    "    (0.1, '0.517-#48'),\n",
    "    #\n",
    "    ## 2nd-generation\n",
    "    ###\n",
    "    (2.0, '0.859-d5-#95'), # ema.9 0.885 | RETEST WITH NEW TTA256 [+] -> 0.871\n",
    "    (1.5, 'dh5-#98'),      # ema.9 0.841 | RETEST WITH NEW TTA64  [+]\n",
    "    (1.5, 'r3-#100'),      # ema.9 0.845 | RETEST WITH NEW TTA64  [-]\n",
    "    (2.0, 'R3-#103'),      # ema.9 0.877 | RETEST WITH NEW TTA128 [+]\n",
    "    (1.0, 'c6-#105'),      # ema.9 0.791 | RETEST WITH NEW TTA32  [+]\n",
    "    \n",
    "#     # 3rd-generation\n",
    "#     ## \n",
    "#     (0.75, 'dc1-#131'), # TTA64 | ema.9 0.9126\n",
    "#     (1.0, 'dhc1-#132'), # TTA32 | ema.9 0.8822\n",
    "#     (1.0, 'D1-#135'),   # TTA32 | ema.9 ~0.89\n",
    "#     (1.0, 'Rc1-#133'),  # TTA32 | ema.9 0.9004\n",
    "#     (1.0, 'Z1-#134'),   # TTA32 | ema.9 0.8995\n",
    "#     (0.5, 'cc1-#137'),  # TTA32 | ema.9 0.7948\n",
    "]\n",
    "weights, dirs = zip(*proba_dirs)\n",
    "P = avg_proba_paths(*map(lambda p: '../submissions/{0}/proba.csv'.format(p), dirs), \n",
    "              weights=list(weights))\n",
    "proba_to_subm(P, '../submissions/averaged_gen12.csv')\n",
    "save_proba(P, '../submissions/proba.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generation 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "    # D121 256x256 d=0 weighted\n",
    "    '0.960-d-#181', \n",
    "#     # BEST FROM D121 d=0.3\n",
    "#     '0.949-dw-0.3-#194',\n",
    "#     # D121 512x512 d=0.1 (unw.)\n",
    "#     '0.946-d-512-#191',\n",
    "#     # R50 512x512 d=0.2 (unw.)\n",
    "#     'R-512-#189',\n",
    "#     # R50 256x256 d=0 (unw.)\n",
    "#     ('R-#190'),\n",
    "#     # TODO: C2 * 0.25"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## generation 5 (bootstrapping) $\\rightarrow$ 0.973"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.00883252,  0.01456236,  0.02400928,  0.03958461,  0.06526399,\n",
       "         0.10760213,  0.17740591,  0.2924929 ,  0.48223927,  0.79507815]])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax(np.arange(10))**0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.49823845,  0.49823845,  0.49823845,  0.49823845,  0.49823845,\n",
       "         0.49823845,  0.49823845,  0.49823845,  0.49823845,  0.49823845]])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax(np.arange(10)/2.) / (softmax(np.arange(10))**0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def w_median(x, w, axis):\n",
    "    x_hat = len(x) / float(sum(w)) * (x * w[:, np.newaxis, np.newaxis])\n",
    "    return np.median(x_hat, axis=axis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.5"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w_median(np.arange(4), np.ones(4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.3333333333333333"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w_median(np.arange(4), np.array([2,2,1,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.6666666666666665"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w_median(np.arange(4), np.array([1,1,2,2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1) Arithmetic (proba)    \t0.9833\n",
      "2) Geometric (avg. logits)\t0.9792\n",
      "3) proba ** 0.5\t\t\t0.9812\n",
      "3) proba ** 2\t\t\t0.9812\n",
      "[x] g(logits)\t\t\t0.9708\n",
      "5) softmax(g(logits))\t\t0.9771\n",
      "6) median(proba)\t\t0.9771\n",
      "7) median(logits)\t\t0.9708\n",
      "8) w_median(proba)\t\t0.9771\n",
      "9) w_median(logits)\t\t0.9750\n"
     ]
    }
   ],
   "source": [
    "# (d121/10., ),\n",
    "# (d121/10., ),\n",
    "# (D201/2., 'Dp-0.3-fc@10(0.9354)-#209'),\n",
    "\n",
    "proba_dirs = [\n",
    "    (1./3., [\n",
    "        (1., '0.965-dp-0.1-final-no-weighted-#220'),\n",
    "        (1., '0.969-dp-0.1-#204'),\n",
    "        (1., '0.970-dp-0.2-fc-#208'),\n",
    "    ]),\n",
    "    (1./2., [\n",
    "        (1., 'Dp-0.15-fc(0.9500)-#213'),\n",
    "        (1., 'Dp-0.25-fc(0.9500)-#224'),\n",
    "    ]),\n",
    "    (1./2., [\n",
    "        (1., 'R152p-0.15-fc(0.9458)-#216'),\n",
    "        (1., 'R152p-0.25-fc(0.9521)-#227'),\n",
    "    ]),\n",
    "    (1./7., [\n",
    "        (1., 'RX64p-0.1(0.9521)-#217'),\n",
    "        (1., 'RX64p-0.2-fc(0.9583)-#226'),\n",
    "        (1., 'RXp-0.1(0.9500)-#215'),\n",
    "        (1., 'RXp-0.2-fc(0.9479)-#225'),\n",
    "\n",
    "        (1., 'RX-bagg-1-a21(0.9396)-#257'),\n",
    "        (1., 'RX-bagg-1-a10(0.9333)-#258'),\n",
    "        (1., 'RX-bagg-1-a19(0.9417)-#259'),\n",
    "    ]),\n",
    "    (1./19., [\n",
    "        (1., 'dp-final(0.9604)-#221'),\n",
    "        (1., 'dp-0.05-final(0.9500)-#223'),\n",
    "        (1., 'dp-0.1-bs(0.9521)-#228'),\n",
    "        (1., 'dp-0.1-final(0.9563)-#219'),\n",
    "        (1., 'dp-0.1-adam(0.9542)-#218'),\n",
    "        (1., 'dp-0.1-final-bag-1'),\n",
    "        (1., 'dp-0.1-final-bag-2'),\n",
    "        (1., 'dp-0.1-final-bag-3'),\n",
    "        (1., 'dp-0.1-final-bag-4'),\n",
    "        (1., 'dp-0.1-final-bag-5'),\n",
    "        (1., 'dp-0.1-final-bag-6'),\n",
    "        (1., 'dp-0.1-final-bag-7'),\n",
    "        \n",
    "        (1., 'd121-bagg-10(0.9437)-#254'),\n",
    "        (1., 'd121-bagg-2(0.9437)-#246'),\n",
    "        (1., 'd121-bagg-3(0.9542)-#247'),\n",
    "        (1., 'd121-bagg-4(0.9313)-#248'),\n",
    "        (1., 'd121-bagg-6(0.9417)-#250'),\n",
    "        (1., 'd121-bagg-8(0.9458)-#252'),\n",
    "        (1., 'd121-bagg-9(0.9313)-#253'),\n",
    "    ]),\n",
    "]\n",
    "\n",
    "proba_dirs_normalized = []\n",
    "for meta_w, dirs in proba_dirs:\n",
    "    for w, d in dirs:\n",
    "        proba_dirs_normalized.append((meta_w * w, d))\n",
    "proba_dirs = proba_dirs_normalized\n",
    "\n",
    "y_val = np.load('../submissions/{0}/y_val.npy'.format(proba_dirs[0][1]))\n",
    "def val_acc(z):\n",
    "    return sum(z.argmax(axis=1) == y_val)/float(len(y_val))\n",
    "\n",
    "X = []\n",
    "\n",
    "for w, path in proba_dirs:\n",
    "    logits = np.load('../submissions/{0}/logits_val.npy'.format(path))\n",
    "    X.append((w, logits))\n",
    "\n",
    "def g(x):\n",
    "    return np.sqrt(np.abs(x))*np.sign(x)\n",
    "    \n",
    "w, X = zip(*X)\n",
    "X = np.asarray(X) # logits\n",
    "Y = X.copy()\n",
    "print \"1) Arithmetic (proba)    \\t{0:.4f}\".format(val_acc(np.average([softmax(y) for y in Y], axis=0, weights=w)))\n",
    "print \"2) Geometric (avg. logits)\\t{0:.4f}\".format(val_acc(np.average([y for y in Y], axis=0, weights=w)))\n",
    "print \"3) proba ** 0.5\\t\\t\\t{0:.4f}\".format(val_acc(np.average([softmax(y)**0.5 for y in Y], axis=0, weights=w)))\n",
    "print \"3) proba ** 2\\t\\t\\t{0:.4f}\".format(val_acc(np.average([softmax(y)**2. for y in Y], axis=0, weights=w)))\n",
    "print \"[x] g(logits)\\t\\t\\t{0:.4f}\".format(val_acc(np.average([g(y) for y in Y], axis=0, weights=w)))\n",
    "print \"5) softmax(g(logits))\\t\\t{0:.4f}\".format(val_acc(np.average([softmax(g(y)) for y in Y], axis=0, weights=w)))\n",
    "print \"6) median(proba)\\t\\t{0:.4f}\".format(val_acc(np.median([softmax(y) for y in Y], axis=0)))\n",
    "print \"7) median(logits)\\t\\t{0:.4f}\".format(val_acc(np.median([y for y in Y], axis=0)))\n",
    "print \"8) w_median(proba)\\t\\t{0:.4f}\".format(val_acc(w_median([softmax(y) for y in Y], np.asarray(w), axis=0)))\n",
    "print \"9) w_median(logits)\\t\\t{0:.4f}\".format(val_acc(w_median(Y, np.asarray(w), axis=0)))\n",
    "\n",
    "weights, dirs = zip(*proba_dirs)\n",
    "probas = map(lambda d: get_proba('../submissions/{0}/proba.csv'.format(d)), dirs)\n",
    "P = w_median(probas, np.asarray(weights), axis=0)\n",
    "# P = avg_proba_paths(*map(lambda p: '../submissions/{0}/proba.csv'.format(p), dirs), weights=list(weights))\n",
    "proba_to_subm(P, '../submissions/median-submission.csv');\n",
    "save_proba(P, '../submissions/proba.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## new data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "proba_dirs = [\n",
    "    (1./4., [\n",
    "        (1., 'dt-0.25-t16-dc0.12(0.9529)-#305'),    # DONE FIVE-CROP+ROT\n",
    "        (1., 'dt-0.25-dc0.08-fc(0.9389)-#311'),     # DONE FIVE-CROP+ROT\n",
    "        (1., 'dt-0.2-dd0.96-dc0.08(0.9516)-#296'),  # DONE FIVE-CROP+ROT\n",
    "        (1., 'dt-0.2-t4-fc(0.9592)-#308'),          # DONE FIVE-CROP+ROT\n",
    "#         (1., 'dt-0.2-no-w-cs224(0.9274)-#310'),\n",
    "    ]),\n",
    "    (1., [\n",
    "#         (1., 'Dt-0.3-dc0.12(0.9261)-#302'),\n",
    "#         (1., 'Dt-0.3-fca(0.9299)-#307'),\n",
    "        (1., 'Dt-0.4-fc(0.9478)-#301'),             # DONE FIVE-CROP+ROT\n",
    "    ]),\n",
    "    (1., [\n",
    "        (1., 'R152t-0.35-fc(0.9350)-#300'),         # DONE FIVE-CROP+ROT\n",
    "    ]),\n",
    "    (1./3., [\n",
    "        (1., 'RXt-0.2(0.9529)-#309'),               # DONE FIVE-CROP+ROT\n",
    "#         (1., 'RX64t-0.2(0.9287)-#303'),\n",
    "        (1., 'RX64t-0.2-fc(0.9401)-#304'),          # DONE FIVE-CROP+ROT\n",
    "#         (1., 'RXt-0.3-fc(0.9312)-#306'),\n",
    "        (1., 'RXt-0.25(0.9363)-#312')\n",
    "    ]),\n",
    "]\n",
    "\n",
    "proba_dirs_normalized = []\n",
    "for meta_w, dirs in proba_dirs:\n",
    "    for w, d in dirs:\n",
    "        proba_dirs_normalized.append((meta_w * w, d))\n",
    "proba_dirs = proba_dirs_normalized\n",
    "\n",
    "weights, dirs = zip(*proba_dirs)\n",
    "P = avg_proba_paths(*map(lambda p: '../submissions/{0}/proba.csv'.format(p), dirs), weights=list(weights))\n",
    "# proba_to_subm(P, '../submissions/#317-submission.csv');\n",
    "# save_proba(P, '../submissions/proba.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## THE FINALEST SUBMISSIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2640, 10)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proba_dirs = [\n",
    "    (1./3., [\n",
    "#         (1., 'dt-0.25-t16-dc0.12(0.9529)-#305'),\n",
    "#         (1., 'dt-0.25-dc0.08-fc(0.9389)-#311'),\n",
    "#         (1., 'dt-0.2-dd0.96-dc0.08(0.9516)-#296'),\n",
    "#         (1., 'dt-0.2-t4-fc(0.9592)-#308'),\n",
    "        (1., 'dx-0.2-fc(0.9592)-#322'),\n",
    "        (1., 'dx-0.25(0.9554)-#323'),\n",
    "        (1., 'dx-0.3-fc(0.9541)-#324')\n",
    "    ]),\n",
    "    (1./2., [\n",
    "        (1., 'Dt-0.4-fc(0.9478)-#301'),\n",
    "        (1., 'Dx-0.35-fc(0.9567)-#317'),\n",
    "    ]),\n",
    "    (1., [\n",
    "#         (1., 'R152t-0.35-fc(0.9350)-#300'),\n",
    "        (1., 'R152x-0.35-fc(0.9439)-#318')\n",
    "    ]),\n",
    "    (1./2., [\n",
    "        (1., 'RXt-0.2(0.9529)-#309'),\n",
    "        (1., 'RX64t-0.2-fc(0.9401)-#304')\n",
    "    ])\n",
    "]\n",
    "\n",
    "proba_dirs_normalized = []\n",
    "for meta_w, dirs in proba_dirs:\n",
    "    for w, d in dirs:\n",
    "        proba_dirs_normalized.append((meta_w * w, d))\n",
    "proba_dirs = proba_dirs_normalized\n",
    "\n",
    "weights, dirs = zip(*proba_dirs)\n",
    "P = avg_proba_paths(*map(lambda p: '../submissions/{0}/proba-5crop.csv'.format(p), dirs), weights=list(weights))\n",
    "\n",
    "# P = avg_proba_paths(*map(lambda p: '../submissions/{0}/proba-5crop.csv'.format(p), dirs))\n",
    "P.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "proba_to_subm(P, '../submissions/#327-submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "79"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(P[P.max(axis=1) < 0.7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 267.22822722  224.9312485   272.08140485  282.31885467  269.99548979\n",
      "  271.67557961  263.33269105  255.22620103  267.40246744  265.80783584]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEACAYAAAC9Gb03AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEJhJREFUeJzt3X+MZWV9x/H3B7dalXalVnYrq2xVimsTu5IUUNK4BFPB\nJl3TBIKaKP5ISKRKJGkE/ln64w9tgq3GGmJFshgUkP4AGytI1mljUkELhLW70m0t7i50R62gQRMD\n8u0fc9Yd1tmZ2Z25c5595v1Kbvbcc88953Mvw+c+89x77qSqkCT164SxA0iSJsuil6TOWfSS1DmL\nXpI6Z9FLUucseknq3IJFn+Q5Se5Jcn+SnUm2Des3JvlakoeSfC7JmmH9s5PcnGRPkn9L8tJJPwhJ\n0pEtWPRV9VPg3Kp6DbAZuCDJWcCHgWur6nTgceDdw13eDfygqk4D/hr4y4kklyQtyqKmbqrqJ8Pi\nc4A1QAHnAn83rN8OvHlY3jpcB7gNOG9ZkkqSjsmiij7JCUnuBw4AXwb+G3i8qp4eNtkPnDIsnwLs\nA6iqnwGPJ/m1ZU0tSVq0xY7onx6mbjYAZwKb5tps+DeHrc+s2yRJK2zN0WxcVT9K8i/A2cALkpww\njOo3AI8Om+0HXgI8muRZwK9W1WOH7yuJ5S9Jx6CqDh9Qz2sxn7r59SRrh+XnAm8AdgFfAS4cNnsH\ncPuwfMdwneH2HfOE9VLFtm3bRs/QysXnwufC52L+y7FYzIj+N4DtSU5g5oXhlqr6YpLdwM1J/hy4\nH7h+2P564DNJ9gD/B1x8TMkkSctiwaKvqp3AGXOs/x/grDnW/xS4aFnSSZKWzDNjG7Bly5axIzTD\n5+IQn4tDfC6WJsc657PkAyc11rEl6XiVhFruN2MlScc3i16SOmfRS1LnLHpJ6pxFL0mds+glqXMW\nvSR1zqLXz61fv5EkE7+sX79x7IcqrSqeMKWfS1bqG6VzzF/OJK12njAlSfoFFr0kdc6il6TOWfSS\n1DmLXpI6Z9E3YKU+1uhHG6XVyY9XNmDlPtYI83200Y9XHrJ+/Uamp78z8eOsW3cqBw48PPHjqB/H\n8vFKi74BFv0hK1WwMH/JtvBcSHOx6I9TFv0YGVrJYdHr6Bx3J0w5Ly1pMXwfa2lGHdG3MHJrgaPY\nMTK0kqPtn81WtPJz0cJ7N8fd1E0L/+Fa0MoPcQvl5nOhufhz8cwMx9XUjaQjc7pCy8URfQMcrYyR\noZUcLWSYP0cLWnkuWvm5cEQvSXoGi16SOmfRS1LnFiz6JBuS7EiyK8nOJO8b1m9Lsj/JfcPl/Fn3\nuSrJniS7k/z+JB+AJGl+axaxzVPAFVX1QJITgX9P8uXhto9U1Udmb5xkE3ARsAnYANyd5DRPg5WO\nTy18dlxLs2DRV9UB4MCw/ESS3cApw81zvfO7Fbi5qp4CHk6yBzgTuGd5IktaSTMlP/lx2vT0UX2Q\nREfhqObok2wENnOotC9L8kCSTyVZO6w7Bdg3626PcOiFQZK0whZd9MO0zW3A5VX1BPAJ4OVVtZmZ\nEf+1Bzed4+5O20jSSBYzR0+SNcyU/Geq6naAqvrerE3+FvjCsLwfeMms2zYAj86952tmLW8ZLpKk\ng6amppiamlrSPhZ1ZmySG4HvV9UVs9atH+bvSfIB4Her6q1JXgXcBJzFzJTNl4FfeDPWM2MP8ay/\nMTK0kqOFDK3kaCFDKzmW98zYBUf0Sc4B3gbsTHI/M4/yauCtSTYDTwMPA5cCVNWuJLcCu4Angff6\niRtJGo/fddMARytjZGglRwsZWsnRQoZWcvhdN5Kko2DRS1LnLHpJ6pxFL0mds+glqXMWvSR1zqKX\npM5Z9JLUOYtekjpn0UtS5yx6SeqcRS9JnbPoJalzFr0kdW7VF/369RtJMvHL+vUbx36oklapVf99\n9K18v7TPxUpnaCVHCxlaydFChlZy+H30kqSjYNFLUucseknqnEUvSZ2z6CWpcxa9JHXOopekzln0\nktQ5i16SOmfRS1LnLHpJ6pxFL0mds+glqXMLFn2SDUl2JNmVZGeS9w/rT0pyV5KHktyZZO2s+3ws\nyZ4kDyTZPMkHIEma32JG9E8BV1TVq4DXApcleSVwJXB3VZ0O7ACuAkhyAfDyqjoNuBS4biLJJUmL\nsmDRV9WBqnpgWH4C2A1sALYC24fNtg/XGf69cdj+HmBtknXLnFuStEhHNUefZCOwGfgasK6qpmHm\nxQA4edjsFGDfrLs9MqyTJI1gzWI3THIicBtweVU9MfMXoubedI51R9j2mlnLW4aLJOmgqakppqam\nlrSPRf0pwSRrgH8C/rmqPjqs2w1sqarpJOuBr1TVpiTXDcu3DNt9C3j9wdH/rH36pwRXPEMrOVrI\n0EqOFjK0kqOFDK3kGOdPCX4a2HWw5Ad3AJcMy5cAt89a//Yh0NnA44eXvCRp5Sw4ok9yDvCvwE5m\nXsoKuBq4F7gVeAmwF7iwqh4f7vNx4Hzgx8A7q+q+OfbriH7FM7SSo4UMreRoIUMrOVrI0EqO5R3R\nL2rqZhIs+jEytJKjhQyt5GghQys5WsjQSo5xpm4kSccpi16SOmfRS1LnLHpJ6pxFL0mds+glqXMW\nvSR1zqKXpM5Z9JLUOYtekjpn0UtS5yx6SeqcRS9JnbPoJalzFr0kdc6il6TOWfSS1DmLXpI6Z9FL\nUucseknqnEUvSZ2z6CWpcxa9JHXOopekzln0ktQ5i16SOmfRS1LnLHpJ6tyCRZ/k+iTTSR6ctW5b\nkv1J7hsu58+67aoke5LsTvL7kwouSVqcxYzobwDeOMf6j1TVGcPlSwBJNgEXAZuAC4BPJMmypZUk\nHbUFi76qvgo8NsdNcxX4VuDmqnqqqh4G9gBnLimhJGlJljJHf1mSB5J8KsnaYd0pwL5Z2zwyrJMk\njWTNMd7vE8CfVVUl+QvgWuA9zD3KryPv5ppZy1uGiyTpoKmpKaamppa0j1TN08MHN0pOBb5QVa+e\n77YkVwJVVR8ebvsSsK2q7pnjfjXva8CyCkd6nDNvIaxEjhYytJKjhQyt5GghQys5WsjQSo75M1TV\nUb33udipmzBrtJ5k/azb/gj45rB8B3Bxkmcn+U3gFcC9RxNIkrS8Fpy6SfJZZuZUXphkL7ANODfJ\nZuBp4GHgUoCq2pXkVmAX8CTw3lrMrwySpIlZ1NTNRA7s1M0IGVrJ0UKGVnK0kKGVHC1kaCXHOFM3\nkqTjlEUvSZ2z6CWpcxa9JHXOopekzln0ktQ5i16SOmfRS1LnLHpJ6pxFL0mds+glqXMWvSR1zqKX\npM5Z9JLUOYtekjpn0UtS5yx6SeqcRS9JnbPoJalzFr0kdc6il6TOWfSS1DmLXpI6Z9FLUucseknq\nnEUvSZ2z6CWpcxa9JHVuwaJPcn2S6SQPzlp3UpK7kjyU5M4ka2fd9rEke5I8kGTzpIJLkhZnMSP6\nG4A3HrbuSuDuqjod2AFcBZDkAuDlVXUacClw3TJmlSQdgwWLvqq+Cjx22OqtwPZheftw/eD6G4f7\n3QOsTbJueaJKko7Fsc7Rn1xV0wBVdQA4eVh/CrBv1naPDOskSSNZs8z7yxzr6sibXzNrectwkSQd\nNDU1xdTU1JL2kap5evjgRsmpwBeq6tXD9d3AlqqaTrIe+EpVbUpy3bB8y7Ddt4DXHxz9H7bPmvc1\nYFmFIz3OJKxMjhYytJKjhQyt5GghQys5WsjQSo75M1TVXIPqI1rs1E145mj9DuCSYfkS4PZZ698+\nhDkbeHyukpckrZwFp26SfJaZOZUXJtkLbAM+BHw+ybuAvcCFAFX1xSRvSvJfwI+Bd04quCRpcRY1\ndTORAzt1M0KGVnK0kKGVHC1kaCVHCxlayTHO1I0k6Thl0UtS5yx6SeqcRS9JnbPoJalzFr0kdc6i\nl6TOWfSS1DmLXpI6Z9FLUucseknqnEUvSZ2z6CWpcxa9JHXOopekzln0ktQ5i16SOmfRS1LnLHpJ\n6pxFL0mds+glqXMWvSR1zqKXpM5Z9JLUOYtekjpn0UtS5yx6SeqcRS9JnVuzlDsneRj4IfA08GRV\nnZnkJOAW4FTgYeCiqvrhEnNKko7RUkf0TwNbquo1VXXmsO5K4O6qOh3YAVy1xGNIkpZgqUWfOfax\nFdg+LG8H3rzEY0iSlmCpRV/AnUm+nuQ9w7p1VTUNUFUHgBct8RiSpCVY0hw98LqqOpDkRcBdSR5i\npvwlSY1YUtEPI3aq6ntJ/hE4E5hOsq6qppOsB7575D1cM2t5y3CRJB00NTXF1NTUkvaRqmMbgCd5\nHnBCVT2R5PnAXcCfAucBP6iqDyf5IHBSVV05x/1r5Qb/4UiPMwkrk6OFDK3kaCFDKzlayNBKjhYy\ntJJj/gxVlaPZ21JG9OuAf5gpbNYAN1XVXUm+Adya5F3AXuDCJRxDkrRExzyiX/KBHdGPkKGVHC1k\naCVHCxlaydFChlZyLO+I3jNjJalzFr0kdc6il6TOWfSS1DmLXpI6Z9FLUucseknqnEUvSZ2z6CWp\ncxa9JHXOopekzln0ktQ5i16SOmfRS1LnLHpJ6pxFL0mds+glqXMWvSR1zqKXpM5Z9JLUOYtekjpn\n0UtS5yx6SeqcRS9JnbPoJalzFr0kdc6il6TOWfSS1LmJFX2S85N8K8l/JvngpI4jSZrfRIo+yQnA\nx4E3Ar8NvCXJKydxLEnS/CY1oj8T2FNV36mqJ4Gbga0TOpYkaR6TKvpTgH2zru8f1kmSVtikij5z\nrKsJHUuSNI81E9rvfuCls65vAB79xc3mej2YjGS+Y61MjhYytJKjhQyt5GghQys5WsjQSo75Mxzl\nvqqWf6Cd5FnAQ8B5wP8C9wJvqardy34wSdK8JjKir6qfJflj4C5mpoeut+QlaRwTGdFLktoxypmx\nnkw1I8mGJDuS7EqyM8n7x840piQnJLkvyR1jZxlbkrVJPp9kd5L/SHLW2JnGkuQDSb6Z5MEkNyV5\n9tiZVkqS65NMJ3lw1rqTktyV5KEkdyZZu9B+VrzoPZnqGZ4CrqiqVwGvBS5bxc8FwOXArrFDNOKj\nwBerahPwO8CqnPpM8mLgfcAZVfVqZqabLx431Yq6gZmunO1K4O6qOh3YAVy10E7GGNF7MtWgqg5U\n1QPD8hPM/M+8Ks83SLIBeBPwqbGzjC3JrwC/V1U3AFTVU1X1o5FjjelZwPOTrAGex5yf4OtTVX0V\neOyw1VuB7cPyduDNC+1njKL3ZKo5JNkIbAbuGTfJaP4K+BM83wLgZcD3k9wwTGV9Mslzxw41hqp6\nFLgW2As8AjxeVXePm2p0J1fVNMwMFoEXLXSHMYrek6kOk+RE4Dbg8mFkv6ok+QNgevjtJqzkB6bb\ntAY4A/ibqjoD+Akzv66vOklewMwI9lTgxcCJSd46bqrjzxhFv8iTqVaH4dfR24DPVNXtY+cZyTnA\nHyb5NvA54NwkN46caUz7gX1V9Y3h+m3MFP9q9Abg21X1g6r6GfD3wOtGzjS26STrAJKsB7670B3G\nKPqvA69Icurw7vnFwGr+lMWngV1V9dGxg4ylqq6uqpdW1cuY+XnYUVVvHzvXWIZfy/cl+a1h1Xms\n3jep9wJnJ/nlzJwqeh6r743pw3/LvQO4ZFh+B7DgAHFSX4FwRJ5MdUiSc4C3ATuT3M/MFNbVVfWl\ncZOpAe8HbkryS8C3gXeOnGcUVXVvktuA+4Enh38/OW6qlZPks8AW4IVJ9gLbgA8Bn0/yLmZeCC9c\ncD+eMCVJffNPCUpS5yx6SeqcRS9JnbPoJalzFr0kdc6il6TOWfSS1DmLXpI69/8N0mkuwK5rpAAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f8143dbb690>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "P_sum = np.sum(P, axis=(0,))\n",
    "print P_sum\n",
    "plt.bar(range(10), P_sum);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_distribution(ids, classes=None, prediction_probabilities=None):\n",
    "    if classes is None:\n",
    "        classes = [get_class(idx.split('/')[-2]) for idx in ids]\n",
    "    classes=np.array(classes)\n",
    "    classes_count = np.bincount(classes)\n",
    "    threshold = 0.7\n",
    "    poor_prediction_probabilities = 0\n",
    "    for class_idx, (class_name, class_count) in enumerate(zip(range(10), classes_count)):\n",
    "        if prediction_probabilities is not None:\n",
    "            prediction_probabilities_this_class = prediction_probabilities[classes == class_idx, class_idx]\n",
    "            poor_prediction_probabilities_this_class = (prediction_probabilities_this_class < threshold ).sum()\n",
    "            poor_prediction_probabilities += poor_prediction_probabilities_this_class\n",
    "            poor_prediction_probabilities_this_class /= prediction_probabilities_this_class.size\n",
    "        print('{:>22}: {:5d} ({:04.1f}%)'.format(class_name, class_count, 100. * class_count / len(classes)) + \\\n",
    "            (' Poor predictions: {:04.1f}%'.format(100 * poor_prediction_probabilities_this_class) if prediction_probabilities is not None else ''))\n",
    "    if prediction_probabilities is not None:\n",
    "        print(\"                                Total poor predictions: {:04.1f}% (threshold = {:03.1f})\".format( \\\n",
    "            100. * poor_prediction_probabilities / classes.size, threshold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                     0:   269 (10.2%) Poor predictions: 00.0%\n",
      "                     1:   229 (08.7%) Poor predictions: 00.0%\n",
      "                     2:   275 (10.4%) Poor predictions: 00.0%\n",
      "                     3:   273 (10.3%) Poor predictions: 00.0%\n",
      "                     4:   265 (10.0%) Poor predictions: 00.0%\n",
      "                     5:   274 (10.4%) Poor predictions: 00.0%\n",
      "                     6:   263 (10.0%) Poor predictions: 00.0%\n",
      "                     7:   260 (09.8%) Poor predictions: 00.0%\n",
      "                     8:   268 (10.2%) Poor predictions: 00.0%\n",
      "                     9:   264 (10.0%) Poor predictions: 00.0%\n",
      "                                Total poor predictions: 02.3% (threshold = 0.7)\n"
     ]
    }
   ],
   "source": [
    "print_distribution(None, map(int, unhot(one_hot_decision_function(P))), P)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## equalizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "P_sum = np.sum(P, axis=(0,))\n",
    "prediction_probabilities = P\n",
    "\n",
    "items_per_class = len(prediction_probabilities) / 10\n",
    "\n",
    "submission = np.zeros(2640, dtype=np.int)\n",
    "\n",
    "for class_idx in np.argsort(P_sum)[::-1]:\n",
    "    largest_idx = np.argpartition(P[:,class_idx], -items_per_class)[-items_per_class:]\n",
    "    P[largest_idx] = 0.\n",
    "    for i in largest_idx:\n",
    "        submission[i] = class_idx\n",
    "\n",
    "len(submission)\n",
    "subm(submission, '../submissions/#327-equalized.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generate logits from 0.975 ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dirs = [\n",
    "    '0.965-dp-0.1-final-no-weighted-#220',\n",
    "    '0.969-dp-0.1-#204',\n",
    "    '0.970-dp-0.2-fc-#208',\n",
    "    'dp-0.1-final-bag-1',\n",
    "    'dp-0.1-final-bag-2',\n",
    "    'dp-0.1-final-bag-3',\n",
    "    'dp-0.1-final-bag-4',\n",
    "    'dp-0.1-final-bag-5',\n",
    "    'dp-0.1-final-bag-6',\n",
    "    'dp-0.1-final-bag-7',\n",
    "]\n",
    "\n",
    "logits_all = []\n",
    "for path in dirs:\n",
    "    logits = np.load('../submissions/{0}/logits_train.npy'.format(path))\n",
    "    logits_all.append(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logits = np.average(logits_all, axis=0)\n",
    "np.save('../data/logits_train.npy', logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## generation 4+5th"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dirs = [\n",
    "    # w/o pseudo-labels\n",
    "#     '0.960-d-#181',\n",
    "    # w/ pseudo-labels\n",
    "    '0.965-dp-0.1-final-no-weighted-#220',\n",
    "    '0.969-dp-0.1-#204',\n",
    "    '0.970-dp-0.2-fc-#208',\n",
    "    # <ALL FROM \n",
    "]\n",
    "P = avg_proba_paths(*map(lambda p: '../submissions/{0}/proba.csv'.format(p), dirs))\n",
    "proba_to_subm(P, '../submissions/#236-submission.csv');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# P1 = get_proba('../submissions/0.977-#273/proba.csv')\n",
    "# P2 = get_proba('../submissions/0.966-#296/proba.csv')\n",
    "P = avg_proba_paths('../submissions/#315/proba.csv', '../submissions/0.977-#273/proba.csv')\n",
    "proba_to_subm(P, '../submissions/#316-submission.csv');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ($\\approx$blending) $\\rightarrow$ 0.968 (overfit to val. set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((480, 31), (480,))"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dirs = [\n",
    "    # w/o pseudo-labels\n",
    "#     '0.960-d-#181',\n",
    "    # w/ pseudo-labels\n",
    "    '0.965-dp-0.1-final-no-weighted-#220',\n",
    "    '0.969-dp-0.1-#204',\n",
    "    '0.970-dp-0.2-fc-#208',\n",
    "    # <ALL FROM \n",
    "]\n",
    "\n",
    "X = []\n",
    "manip = np.load('../submissions/{0}/manip_val.npy'.format(dirs[0]))\n",
    "y = np.load('../submissions/{0}/y_val.npy'.format(dirs[0]))\n",
    "for d in dirs:\n",
    "#     X_b = softmax(np.load('../submissions/{0}/logits_val.npy'.format(d)))\n",
    "    X_b = np.load('../submissions/{0}/logits_val.npy'.format(d))\n",
    "    X_b -= X_b.mean(axis=1)[:, np.newaxis]\n",
    "    X.append(X_b)\n",
    "X = [manip] + X\n",
    "X = np.hstack(X)\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### just average predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.966666666667\n",
      "0.975\n",
      "0.964583333333\n",
      "Avg 0.975\n",
      "Geomean 0.76875\n"
     ]
    }
   ],
   "source": [
    "print sum(X[0].argmax(axis=1) == y) / 480.\n",
    "print sum(X[1].argmax(axis=1) == y) / 480.\n",
    "print sum(X[2].argmax(axis=1) == y) / 480.\n",
    "print \"Avg\", sum(((X[0] + X[1] + X[2])/3.).argmax(axis=1) == y) / 480.\n",
    "print \"Geomean\", sum(avg_proba(X[0], X[1], X[2]).argmax(axis=1) == y) / 480."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVoAAAD3CAYAAABcpJzyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFmpJREFUeJzt3X+QXWV9x/HPZwlLMIkgkQAGkgw/gjWIGJCJWBWBkUgZ\n6KBRUUcQR5hRhAI6oNOKUNsRLYqtRQZFilYaDa2FMo4BTYO1CJKEQBp+OkhCICThV1mYivnx7R/n\n7rK5ufecs3vOuXv25P2a2Zm9d5/7nCdnlg/PnvOc5+uIEACgOn1jPQAAaDqCFgAqRtACQMUIWgCo\nGEELABUjaAGgYgQtAAxj+zrbG2zfn9Lm720/anul7SOy+iRoAWB710s6sdsPbb9P0kERcYikcyRd\nk9UhQQsAw0TEryU9n9LkVEk/aLW9W9IetvdJ65OgBYCRmS7piWGvn2y91xVBCwAj4w7vpe5lQNAC\naIxZs2aF7ZF8PT2Kw6yTdMCw1/tLeirtAwQtgMZYs2aNtm3blvtLUrdrq1bnmask3SLp45Jke56k\nFyJiQ9q4Jozy3wMAtVR0R0LbN0o6VtJU22slXSqpP+k6ro2In9k+yfbvJL0s6ROZfbJNIoCmsB1b\ntmzJ3X7ChAmKiG4z19IwowXQKK1LArVC0AJolDr+lU7QAmgUghYAKkbQAkDFCFoAqBhBCwAVI2gB\noGIs7wKAijGjBYCKEbQAUDGCFgAqRtACQMUIWgCoWB2Dlo2/ATTKCDf+3oHt+bYfsv2I7Ys7/HyG\n7V/Yvs/2EttvyBoTQQugUSIi91c7232Svq2k3PgcSafbfmNbs7+T9E8R8RZJl0v6ataYCFoAjVIk\naCUdLenRiFgTEZslLVRSXny4N0la0jrW0g4/3wFBC6BRCgZteynxddqxlPhKSe+XJNunSZps+3Vp\nY+JmGIBGSbsZduedd+rOO+9M+3ieUuKfl/Rt22dK+pWkJyWl1s+hZhiAxrAdTz75ZO7206dP365m\nWKuq7ZcjYn7r9SVKijJe0eV4kyQ9GBEz0o7DpQMAjVLw0sE9kg62PdN2v6QPKykvPsT2VNuD4fwF\nSd/PGhNBC6BRiizvioitks6VdJuk1ZIWRsSDti+zfXKr2bGSHrb9kKRpkv4ma0xcOgDQGLZjzZo1\nudvPnDmTcuMAMFJ1nDwStAAahaAFgIoRtABQMYIWACpGzTAAqBgzWgCoGEELABUjaAGgYgQtAFSM\noAWAihG0AFAxlncBQMXqOKNlm0QAjVJwP9o8VXAPaFW/XWF7pe33ZY2JoAXQKD2ogvuXkn4cEXMl\nnS7p6qwxEbQAGqUHVXC3SXpt6/s9ldQMS8U1WgCNUvAabacquEe3tblM0m22z5P0GkknZHVK0AJo\nlLSgXbZsmZYvX5728TxVcE+XdH1EfLNVzPGflVxm6IqgBdAoacu75s6dq7lz5w69vvbaa9ubrJM0\nvKLt/pKeamvzSSXXcBURd9meaPv1EfFMt+NyjRZAo1RdBVfSGrUuF9j+E0m7pYWsxIwWQMMUuUYb\nEVttD1bB7ZN03WAVXEn3RMStkj4n6bu2L1ByY+yMrH6pggugMWzHXXfdlbv9vHnzqIILACNVx8kj\nQQugUQhaAKgYQQsAFWP3LgCoGDNaAKgYQQsAFSNoAaBiBC0AVKxxQWt7vqSr9Oqjald0aFO/fzWA\n2ir6pFajgnbYTuTHK9nd5h7bN0fEQ+1tZ86cOfT9Cy+8oD333HO7n69Zs2a0w9jOpEmTMtu8/PLL\nhY8zYUL2adt9990z2wwMDGS2+eUvf5nZ5qyzztrhvfbzXNY53mWXXTLbbN26tZRjLViwILPNokWL\nCh/n8ssvz2zzne98Z4f3BgYGNGXKlKHX69evLzwWKd/v15YtW0o51le+8pXMNl/60pcy2+RZUvWh\nD30o9efz5s3TBRdckNlPGWPptSK7d+XZiRwAeqpozbAqFLl0kGcncgDoqUZdOlC+ncglJX/GDurr\nYwvcXpg4ceJYD6Hx+vv7x3oI497GjRu1ceNGSdvnRBFFgzbr3pPtb0h6j5K8myRp74jYK63PIkGb\nZydySdrhmiyqR9BWb7fddhvrIYx706ZN07Rp0yQl12gXL15cuM8iQZvn3lNEXDis/bmSjsjqt8j0\nMs9O5ADQUz2ogjvc6ZL+JWtMo57RdtuJfLT9AUAZCq46yH3vyfYMSbMkLcnqtNA62oj4uaRDi/QB\nAGUqeI02970nJX/F3xQ5Dlh5KZtePrCQZ/1mnnWg6I5zXL2d9RwvWLBAixYtKvTAgu245ZbuVzBX\nrVqlVatWDb1euHDhdsdrlQ//ckTMb72+RFJ0eRhrhaRPR0Rm7RwewQXQKGmTx8MOO0yHHXbY0OuF\nCxe2Nxm69yRpvZJZ6+ntjWwfKmnPPCErEbQAGqYHVXClJIB3SOluCFoAjVL0cmine08RcWnb68tG\n0idBC6BRmvZkGADUTh03lSFoATQKM1oAqBhBW7E8+3hmrVFs4vrEMpVxjiXOcxrOcTF1DNpR73Vg\ne3/bS2w/YHuV7fPKHBgAjEbT9qPdIunCiFhpe7Kk5bZv61RhAQB6pY4z2iKbyjwt6enW9y/ZflDJ\nhgwELYAx06igHc72LCV7Mt5dRn8AMFqNXN7Vumxwk6TzI+Kl4kMCsDNavXp1Kf3UcUZbqK6M7QlK\nQvaHEXFzOUMCsDOaM2dOKf007WaYJH1f0gMR8a0yBgMARTVqRmv7HZI+Kuk42/faXtEqagYAY6ZR\nM9qI+G9JpayILmvxdZ4Tt7Mu4uYcV49zXA9VV8FttfmgpEslbZN0X0R8LK3PRj0ZBgBVV8G1fbCk\niyW9PSJetP36rH4JWgCNUnB511AVXEmyPVgFd/jzAZ+S9I8R8aIkRcQzWZ0WWnUAAHVT8Bptpyq4\n09vazJZ0qO1f277T9olZY2JGC6BRelAFd4KkgyW9S9IMSf9le87gDLcTghZAo6QF7cMPP6xHHnkk\n7ePrlITnoP2VXKttb/ObiNgm6XHbD0s6RNLybp0StAAaJS1oZ8+erdmzZw+9vvXWW9ub5KmC+++t\n937QuhF2iKTH0sZE0AJolKqr4EbEYtvvtb1ayS6Gn4uI59P6JWgBNEqPquBeJOmivH2WsalMn6Rl\nktZFxCmj6YPF192VtQiec9wd57g38iy7WrRoUU+O02tlzGjPl/SApNeW0BcAFNKovQ6kpJyNpJMk\nfa+c4QBAMY3a66Dlm5I+L2mPEsYCAIU1akZr+88kbYiIlUoW+XZa6AsAPdW0Ge07JJ1i+yRJu0ua\nYvsHEfHxcoYGoOmWLl2qpUuXltpnHWe0LmNQtt8t6aJOqw5s1+9fPY6UdUcc3XGOeyPPaoC+vj5F\nxKj/OrYdV155Ze72F110UaHj5cU6WgCNUscZbSlBGxF3SLqjjL4AoIjGBm1Rf/jDHzLbTJw4sQcj\nKY+d/ddIL3fSf+655zLb7LXXXplt+vqy75/m+XfV6T+G/v7+UvrhEkS6rN+dBQsWlHKcOv1uDapF\n0AJAWQhaAKgYQQsAFSNoAaBiddxUhpphABql6JNhtufbfsj2I7Yv7vDzM2xvtL2i9XVW1piY0QJo\nlKrLjbcsjIjz8vbLjBZAoxSc0Q6VG4+IzZIGy423G9HTZAQtgEbpQblxSTrN9krbP2ltF5uq0KUD\n23so2Yv2MEnbJJ0VEXePtJ/x9jBCWXq5wD3Pwwh51O1GQ54HQ7Zs2ZL687LO8c78MEKdHtZIu3Tw\n+9//Xo8//njax/OUG79F0o0Rsdn2OZJuUHKpoaui12i/JelnEbHA9gRJrynYHwAUkha0s2bN0qxZ\ns4Ze33HHDjsHZJYbbyvE+F1JV2SNadRBa3uKpHdGxJmtg2+R9OJo+wOAMhT8qyuz3LjtfSPi6dbL\nU5WU8kpVZEZ7oKRnbF8v6S1KCjSeHxH/V6BPACik6nLjks6zfYqkzZKek3RmVr9FgnaCpLmSPhMR\ny2xfJekSSZemfwwAqlN1ufGI+KKkL46kzyKrDtZJeiIilrVe36QkeAFgxFavXl1KP3UsZTPqoI2I\nDZKesD279dbxynGtAgA6mTNnTin91DFoi646OE/Sj2zvKukxSZ8oPiQAGL3GbSoTEfdJeltJYwGA\nwhoXtONR1sLqshZV97J6Qt30cvH6znqe6/SAQK+PlaVuD9VIO2HQAmg2ZrQAUDGCFgAqRtACQMUI\nWgCoGEELABUjaAGgYizvAoCKNW5Ga/sCSZ9UUl1hlaRPRMQfR9rPiSeemNnm9ttvz2xzzTXXZLbJ\nquYwZcqUzD4GBgYy2+SRpzpAnjZ5/g/+2GOPZbY58MADM9vsvffemW123XXXUtps3rw5s02dZFVy\nkKQJE8qZ29TpAYG6KRq0tudLukqvbpPYcWNv2x+Q9BNJR0XEirQ+R72pjO03SPqspLkRcbiS0P7w\naPsDgDIU2VRmWBXcEyXNkXS67Td2aDdZSf7dlWdMRYsz7iJp0rAyNk9ltAeASvWoCu5fKylh80qe\nMRXZJvEpSVdKWivpSUkvRMQvRtsfAJSh6iq4to+QtH9E/CzvmIpcOthTSdLPlPQGSZNtf2S0/QFA\nGbZt25b7q4PUKrhObpp8U9JFGZ/ZTpEr8ydIeiwinmsN4N8kHSPpxgJ9AthJlVlhoZunnnpK69ev\nT/t4VhXcKUqu3S5the6+km62fUraDbEiQbtW0jzbE5VcpzheSQVJABixOXPm6IEHihdpSQva/fbb\nT/vtt9/Q6xUrdsjG1Cq4EfGipGmDr23/p6QLI+LetDEVuUb7WyV1wu6VdJ+S6fO1o+0PAMpQ5Bpt\nRGyVNFgFd7WkhYNVcG2f3OlwqvjSgSLiMkmXFekDAMpUdRXctvePy9NnLZ4MW7x4cSn9nH322YX7\nKGuRfFk74Jf1lEuehxHy2LRpUyn9lPWYZC8fEqjLcXqtbtUcsjTuyTAAqBuCFgAqxqYyAFAxZrQA\nUDGCFgAqRtACQMUIWgCoGEELABUbl0Fr+zpJJ0va0NrgW7ZfJ+nHSnbuelzSByPifyscZ62MtwXc\n4xXnuXpNPMd1XN6VZ6+D65XsNj7cJZJ+ERGHSloi6QtlDwwARqPgfrSVyAzaiPi1pOfb3j5V0g2t\n72+Q9OcljwsARqWOQTvaa7TTImKDJEXE07azK/YBQA/U8Rpt0ZphAFCKMjf+LjKjtT3f9kO2H7F9\ncYefn2P7ftv32v5Vp+KN7UYbtBts79M66L6SNo6yHwCQlGz8XYYeVMH9UUQcHhFvlfR1JaVtUuUN\nWmv7zW1vkXRm6/szJN2csx8AqFTVVXAj4qVhLydLylzmkGd5142SjpU01fZaSZdK+qqkRbbPUlLS\nZkFWPwDQCwWXd3Wqgnt0eyPbn5Z0oaRdJWVu/p0ZtBHRrbLtCVmfBYBeK3gzLLUK7rBjXC3patsf\nlvRXevUv/I568mRYUiyyuxdffDGzjylTpmS2ybPDfZ4d+bOMtwXckrR8+fLMNkceeWRmmze/+c2Z\nbVatWpXZJut3Qhp/53k8Lv6v23jKkBa0zzzzjJ599tm0j2dVwW33Y0nXZI2JR3ABNEpa0E6dOlVT\np04dev3oo4+2N0mtgitJtg+OiN+1Xp4s6ZGsMRG0ABqlyKWDiNhqe7AKbp+k6war4Eq6JyJulXSu\n7RMk/VHJw1xnZPVL0AJolKqr4EbEX4y0T4IWQKPU8cmwngdtROS6EYKdC78XKEsdd+9iRgugUZjR\nAkDFdtqgPeaYY4a+X7t2rWbMmLHdz/v6sp8EHt5HN3nWBOZZ69iu05jrrn3MkyZNyvxMnnN80EEH\nZbbJs+a5/TJBp3Ncx/9ghhvN70Wec1yVuv8eH3roodmNcqjj742rHpTt+v2rAdRWRIz6Yr3tOO64\nzCdihyxZsqTQ8fKqfEbbi38EAAyq44yWa7QAGoVVBwBQsTrOaHtaYSFr5/I6sv247ftau6n/dqzH\n0872dbY32L5/2Huvs32b7YdtL7a9x1iOsV2XMV9qe53tFa2v+WM5xuFs7297ie0HbK+yfV7r/dqe\n5w5j/mzr/dqe57LUsWZYz4I2587ldbRN0rER8daI2GFfyhoYj1WKO41Zkr4REXNbXz/v9aBSbJF0\nYUS8SdLbJX2m9btb5/PcPuZzh/33VtfzXIqdOmiVY+fymrJqXFttPFYp7jJmqfNeoGMuIp6OiJWt\n71+S9KCS7fNqe567jHl668e1PM9l2dmDttPO5dO7tK2TkLTY9j22PzXWg8lpuyrFksZLleLP2F5p\n+3t1+jN8ONuzJB0h6S5J+4yH8zxszHe33qr9eS5iZw/aXDuX19AxEXGUpJOU/IL+6VgPqKGulnRQ\nRBwh6WlJ3xjj8ezA9mRJN0k6vzVLrP3vb4cx1/48F7WzB+1Idy6vhdZMRRGxSdJP1aF+UA2NuyrF\nEbEpXv3N/66kt43leNrZnqAksH4YEYPFSGt9njuNue7nuQzbtm3L/dVJjnLjF9he3fqr4HbbB2SN\nqZdBO7Rzue1+JTuX39LD44+Y7de0ZgSyPUnSeyX9z9iOqqPxWKV4uzG3gmrQaarfef6+pAci4lvD\n3qv7ed5hzOPgPBdWZEab86b9CklHtv4q+FclJcdT9WwdbXTZubxXxx+lfST9tPUY8QQl9dxvG+Mx\nbWc8VinuMub32D5CySqPxyWdM2YDbGP7HZI+KmmV7XuVXDL4oqQrJP2kjuc5Zcwfqet5LkvBSwJD\nN+0lyfbgTfuHhvV/x7D2dyk5z6kq3+sAAHrFdhx11FG52y9btmy7bQJsv1/SiRFxduv1xyQdHRHn\ndTneP0haHxF/m3YcngwD0Chpk8eBgQENDAykfTz3TftWCB8p6d1ZYyJoATRKWtBOnjxZkydPHnq9\nfv369ia5btq3ijN+QdK7Ws8FpCJoATRKwcuhecqNv1XSNUouMTybp1OCFkCjFNm9q9tN+7Zy41+T\nNEnJDWdLWhMRqU8FcjMMQGPYjsMPPzx3+/vvv78ZG38DQC/VcfJI0AJoFIIWACpG0AJAxQhaAKgY\nQQsAFaM4IwBUjBktAFSMoAWAihG0AFAxghYAKkbQAkDFCFoAqFgdl3f1sjgjAFSuaLnxHFVw32l7\nue3Ntk/LMyaCFkCj9KAK7holVY9/lHdMXDoA0Cg9qIK7tvWz3AciaAE0SsGgnS7piWGv1ykJ30II\nWgCNkha0r7zyil555ZW0j+eugjsSBC2ARkkL2v7+fvX39w+9fumll9qb5KqCO1IELYBGKbi8K7MK\nbptc9cZYdQCgUYqsOoiIrZIGq+CulrRwsAqu7ZMlyfZRtp+Q9AFJ19helTUmquACaAzbsffee+du\nv2nTJqrgAsBI1XHySNACaBSCFgAqRtACQMXquKkMQQugUZjRAkDFCFoAqBhBCwAVI2gBoGIELQBU\njKAFgIqxvAsAKsaMFgAqVsegZZtEAI3Sgyq4/bYX2n7U9m9sz+jUz3AELYBG6UEV3E9Kei4iDpF0\nlaSvZY2JoAXQKAVntENVcCNis6TBKrjDnSrphtb3N0k6PmtMBC2ARikYtJ2q4E7v1qZVkeEF23ul\njYmbYQAapeDyrjxVcNvbuEOb7TCjBdAka0bYfkPb6zxVcJ+QdIAk2d5F0msj4vm0gxC0ABojImZF\nhEfwtW9bF0NVcG33K6mCe0tbm/+QdEbr+wWSlmSNi0sHANASEVttD1bB7ZN03WAVXEn3RMStkq6T\n9EPbj0p6VkkYp6IKLgBUjEsHAFAxghYAKkbQAkDFCFoAqBhBCwAVI2gBoGIELQBUjKAFgIr9P+E0\nZrtSnKPxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fa3397f3690>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X[:12,1:], cmap='gray', interpolation='none');\n",
    "plt.colorbar();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### grid search logreg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 52 candidates, totalling 520 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 232 tasks      | elapsed:   26.8s\n",
      "[Parallel(n_jobs=-1)]: Done 382 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 520 out of 520 | elapsed:  2.0min finished\n"
     ]
    }
   ],
   "source": [
    "# TODO: for train use solver 'sag', 'saga' everywhere\n",
    "logreg = LogisticRegression(tol=1e-5, max_iter=1000,\n",
    "                            class_weight='balanced')\n",
    "logreg_cv = GridSearchCV(estimator=logreg,\n",
    "                         param_grid=[{'C': np.logspace(-4, 8, 13), 'multi_class': ['ovr'], 'solver': ['liblinear']},\n",
    "                                     {'C': np.logspace(-4, 8, 13), 'multi_class': ['multinomial'], 'solver': ['lbfgs', 'sag', 'saga']}],\n",
    "                         verbose=1,\n",
    "                         n_jobs=-1,\n",
    "                         cv=10)\n",
    "logreg_cv.fit(X, y);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.989583333333\n",
      "{'multi_class': 'multinomial', 'C': 100000.0, 'solver': 'saga'}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_C</th>\n",
       "      <th>param_multi_class</th>\n",
       "      <th>param_solver</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>...</th>\n",
       "      <th>split7_test_score</th>\n",
       "      <th>split7_train_score</th>\n",
       "      <th>split8_test_score</th>\n",
       "      <th>split8_train_score</th>\n",
       "      <th>split9_test_score</th>\n",
       "      <th>split9_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.019597</td>\n",
       "      <td>0.000480</td>\n",
       "      <td>0.975000</td>\n",
       "      <td>0.975470</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>ovr</td>\n",
       "      <td>liblinear</td>\n",
       "      <td>{u'multi_class': u'ovr', u'C': 0.0001, u'solve...</td>\n",
       "      <td>32</td>\n",
       "      <td>0.98</td>\n",
       "      <td>...</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.974419</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.975000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.972727</td>\n",
       "      <td>0.005771</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.023496</td>\n",
       "      <td>0.002900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.034619</td>\n",
       "      <td>0.000422</td>\n",
       "      <td>0.968750</td>\n",
       "      <td>0.972230</td>\n",
       "      <td>0.001</td>\n",
       "      <td>ovr</td>\n",
       "      <td>liblinear</td>\n",
       "      <td>{u'multi_class': u'ovr', u'C': 0.001, u'solver...</td>\n",
       "      <td>41</td>\n",
       "      <td>0.96</td>\n",
       "      <td>...</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.972093</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.970455</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.970455</td>\n",
       "      <td>0.006005</td>\n",
       "      <td>0.000161</td>\n",
       "      <td>0.022185</td>\n",
       "      <td>0.001637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.049029</td>\n",
       "      <td>0.000388</td>\n",
       "      <td>0.972917</td>\n",
       "      <td>0.976860</td>\n",
       "      <td>0.01</td>\n",
       "      <td>ovr</td>\n",
       "      <td>liblinear</td>\n",
       "      <td>{u'multi_class': u'ovr', u'C': 0.01, u'solver'...</td>\n",
       "      <td>36</td>\n",
       "      <td>0.96</td>\n",
       "      <td>...</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.975000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.975000</td>\n",
       "      <td>0.001329</td>\n",
       "      <td>0.000040</td>\n",
       "      <td>0.022028</td>\n",
       "      <td>0.002858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.075200</td>\n",
       "      <td>0.000383</td>\n",
       "      <td>0.972917</td>\n",
       "      <td>0.992595</td>\n",
       "      <td>0.1</td>\n",
       "      <td>ovr</td>\n",
       "      <td>liblinear</td>\n",
       "      <td>{u'multi_class': u'ovr', u'C': 0.1, u'solver':...</td>\n",
       "      <td>36</td>\n",
       "      <td>0.94</td>\n",
       "      <td>...</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.995349</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.993182</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.990909</td>\n",
       "      <td>0.017963</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.017849</td>\n",
       "      <td>0.002013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.082657</td>\n",
       "      <td>0.000396</td>\n",
       "      <td>0.979167</td>\n",
       "      <td>0.997680</td>\n",
       "      <td>1</td>\n",
       "      <td>ovr</td>\n",
       "      <td>liblinear</td>\n",
       "      <td>{u'multi_class': u'ovr', u'C': 1.0, u'solver':...</td>\n",
       "      <td>30</td>\n",
       "      <td>0.94</td>\n",
       "      <td>...</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.997674</td>\n",
       "      <td>0.975</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.997727</td>\n",
       "      <td>0.005767</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.020242</td>\n",
       "      <td>0.001040</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  mean_score_time  mean_test_score  mean_train_score param_C  \\\n",
       "0       0.019597         0.000480         0.975000          0.975470  0.0001   \n",
       "1       0.034619         0.000422         0.968750          0.972230   0.001   \n",
       "2       0.049029         0.000388         0.972917          0.976860    0.01   \n",
       "3       0.075200         0.000383         0.972917          0.992595     0.1   \n",
       "4       0.082657         0.000396         0.979167          0.997680       1   \n",
       "\n",
       "  param_multi_class param_solver  \\\n",
       "0               ovr    liblinear   \n",
       "1               ovr    liblinear   \n",
       "2               ovr    liblinear   \n",
       "3               ovr    liblinear   \n",
       "4               ovr    liblinear   \n",
       "\n",
       "                                              params  rank_test_score  \\\n",
       "0  {u'multi_class': u'ovr', u'C': 0.0001, u'solve...               32   \n",
       "1  {u'multi_class': u'ovr', u'C': 0.001, u'solver...               41   \n",
       "2  {u'multi_class': u'ovr', u'C': 0.01, u'solver'...               36   \n",
       "3  {u'multi_class': u'ovr', u'C': 0.1, u'solver':...               36   \n",
       "4  {u'multi_class': u'ovr', u'C': 1.0, u'solver':...               30   \n",
       "\n",
       "   split0_test_score       ...         split7_test_score  split7_train_score  \\\n",
       "0               0.98       ...                      0.98            0.974419   \n",
       "1               0.96       ...                      0.98            0.972093   \n",
       "2               0.96       ...                      0.98            0.976744   \n",
       "3               0.94       ...                      0.98            0.995349   \n",
       "4               0.94       ...                      0.98            0.997674   \n",
       "\n",
       "   split8_test_score  split8_train_score  split9_test_score  \\\n",
       "0              0.975            0.975000                1.0   \n",
       "1              0.975            0.970455                1.0   \n",
       "2              0.975            0.975000                1.0   \n",
       "3              0.975            0.993182                1.0   \n",
       "4              0.975            1.000000                1.0   \n",
       "\n",
       "   split9_train_score  std_fit_time  std_score_time  std_test_score  \\\n",
       "0            0.972727      0.005771        0.000147        0.023496   \n",
       "1            0.970455      0.006005        0.000161        0.022185   \n",
       "2            0.975000      0.001329        0.000040        0.022028   \n",
       "3            0.990909      0.017963        0.000023        0.017849   \n",
       "4            0.997727      0.005767        0.000015        0.020242   \n",
       "\n",
       "   std_train_score  \n",
       "0         0.002900  \n",
       "1         0.001637  \n",
       "2         0.002858  \n",
       "3         0.002013  \n",
       "4         0.001040  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print logreg_cv.best_score_\n",
    "print logreg_cv.best_params_\n",
    "pd.DataFrame(logreg_cv.cv_results_).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### grid search SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 52 candidates, totalling 520 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 348 tasks      | elapsed:    4.4s\n",
      "[Parallel(n_jobs=-1)]: Done 520 out of 520 | elapsed:    6.5s finished\n"
     ]
    }
   ],
   "source": [
    "svc = SVC(tol=1e-6, max_iter=-1, class_weight='balanced')\n",
    "svc_cv = GridSearchCV(estimator=svc,\n",
    "                      param_grid={'kernel': ['linear', 'rbf'],\n",
    "                                  'C': np.logspace(-4, 8, 13),\n",
    "                                  'decision_function_shape': ['ovr', 'ovo']},\n",
    "                      verbose=1,\n",
    "                      n_jobs=-1,\n",
    "                      cv=10)\n",
    "svc_cv.fit(X, y);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.985416666667\n",
      "{'kernel': 'linear', 'C': 0.01, 'decision_function_shape': 'ovr'}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_C</th>\n",
       "      <th>param_decision_function_shape</th>\n",
       "      <th>param_kernel</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>...</th>\n",
       "      <th>split7_test_score</th>\n",
       "      <th>split7_train_score</th>\n",
       "      <th>split8_test_score</th>\n",
       "      <th>split8_train_score</th>\n",
       "      <th>split9_test_score</th>\n",
       "      <th>split9_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.015002</td>\n",
       "      <td>0.001839</td>\n",
       "      <td>0.945833</td>\n",
       "      <td>0.946771</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>ovr</td>\n",
       "      <td>linear</td>\n",
       "      <td>{u'kernel': u'linear', u'C': 0.0001, u'decisio...</td>\n",
       "      <td>25</td>\n",
       "      <td>0.90</td>\n",
       "      <td>...</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.944186</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.945455</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.943182</td>\n",
       "      <td>0.007096</td>\n",
       "      <td>0.001576</td>\n",
       "      <td>0.035697</td>\n",
       "      <td>0.003537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.039595</td>\n",
       "      <td>0.003307</td>\n",
       "      <td>0.477083</td>\n",
       "      <td>0.923155</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>ovr</td>\n",
       "      <td>rbf</td>\n",
       "      <td>{u'kernel': u'rbf', u'C': 0.0001, u'decision_f...</td>\n",
       "      <td>45</td>\n",
       "      <td>0.32</td>\n",
       "      <td>...</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.893023</td>\n",
       "      <td>0.525</td>\n",
       "      <td>0.886364</td>\n",
       "      <td>0.725</td>\n",
       "      <td>0.956818</td>\n",
       "      <td>0.005923</td>\n",
       "      <td>0.001954</td>\n",
       "      <td>0.103460</td>\n",
       "      <td>0.026829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.010377</td>\n",
       "      <td>0.002087</td>\n",
       "      <td>0.945833</td>\n",
       "      <td>0.946771</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>ovo</td>\n",
       "      <td>linear</td>\n",
       "      <td>{u'kernel': u'linear', u'C': 0.0001, u'decisio...</td>\n",
       "      <td>25</td>\n",
       "      <td>0.90</td>\n",
       "      <td>...</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.944186</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.945455</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.943182</td>\n",
       "      <td>0.002347</td>\n",
       "      <td>0.003091</td>\n",
       "      <td>0.035697</td>\n",
       "      <td>0.003537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.043459</td>\n",
       "      <td>0.002521</td>\n",
       "      <td>0.477083</td>\n",
       "      <td>0.923155</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>ovo</td>\n",
       "      <td>rbf</td>\n",
       "      <td>{u'kernel': u'rbf', u'C': 0.0001, u'decision_f...</td>\n",
       "      <td>45</td>\n",
       "      <td>0.32</td>\n",
       "      <td>...</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.893023</td>\n",
       "      <td>0.525</td>\n",
       "      <td>0.886364</td>\n",
       "      <td>0.725</td>\n",
       "      <td>0.956818</td>\n",
       "      <td>0.018340</td>\n",
       "      <td>0.000169</td>\n",
       "      <td>0.103460</td>\n",
       "      <td>0.026829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.006535</td>\n",
       "      <td>0.000658</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>0.973145</td>\n",
       "      <td>0.001</td>\n",
       "      <td>ovr</td>\n",
       "      <td>linear</td>\n",
       "      <td>{u'kernel': u'linear', u'C': 0.001, u'decision...</td>\n",
       "      <td>23</td>\n",
       "      <td>0.94</td>\n",
       "      <td>...</td>\n",
       "      <td>0.98</td>\n",
       "      <td>0.972093</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.975000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.972727</td>\n",
       "      <td>0.003142</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>0.024438</td>\n",
       "      <td>0.002156</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  mean_score_time  mean_test_score  mean_train_score param_C  \\\n",
       "0       0.015002         0.001839         0.945833          0.946771  0.0001   \n",
       "1       0.039595         0.003307         0.477083          0.923155  0.0001   \n",
       "2       0.010377         0.002087         0.945833          0.946771  0.0001   \n",
       "3       0.043459         0.002521         0.477083          0.923155  0.0001   \n",
       "4       0.006535         0.000658         0.966667          0.973145   0.001   \n",
       "\n",
       "  param_decision_function_shape param_kernel  \\\n",
       "0                           ovr       linear   \n",
       "1                           ovr          rbf   \n",
       "2                           ovo       linear   \n",
       "3                           ovo          rbf   \n",
       "4                           ovr       linear   \n",
       "\n",
       "                                              params  rank_test_score  \\\n",
       "0  {u'kernel': u'linear', u'C': 0.0001, u'decisio...               25   \n",
       "1  {u'kernel': u'rbf', u'C': 0.0001, u'decision_f...               45   \n",
       "2  {u'kernel': u'linear', u'C': 0.0001, u'decisio...               25   \n",
       "3  {u'kernel': u'rbf', u'C': 0.0001, u'decision_f...               45   \n",
       "4  {u'kernel': u'linear', u'C': 0.001, u'decision...               23   \n",
       "\n",
       "   split0_test_score       ...         split7_test_score  split7_train_score  \\\n",
       "0               0.90       ...                      0.96            0.944186   \n",
       "1               0.32       ...                      0.40            0.893023   \n",
       "2               0.90       ...                      0.96            0.944186   \n",
       "3               0.32       ...                      0.40            0.893023   \n",
       "4               0.94       ...                      0.98            0.972093   \n",
       "\n",
       "   split8_test_score  split8_train_score  split9_test_score  \\\n",
       "0              0.950            0.945455              1.000   \n",
       "1              0.525            0.886364              0.725   \n",
       "2              0.950            0.945455              1.000   \n",
       "3              0.525            0.886364              0.725   \n",
       "4              0.950            0.975000              1.000   \n",
       "\n",
       "   split9_train_score  std_fit_time  std_score_time  std_test_score  \\\n",
       "0            0.943182      0.007096        0.001576        0.035697   \n",
       "1            0.956818      0.005923        0.001954        0.103460   \n",
       "2            0.943182      0.002347        0.003091        0.035697   \n",
       "3            0.956818      0.018340        0.000169        0.103460   \n",
       "4            0.972727      0.003142        0.000021        0.024438   \n",
       "\n",
       "   std_train_score  \n",
       "0         0.003537  \n",
       "1         0.026829  \n",
       "2         0.003537  \n",
       "3         0.026829  \n",
       "4         0.002156  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print svc_cv.best_score_\n",
    "print svc_cv.best_params_\n",
    "pd.DataFrame(svc_cv.cv_results_).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clf = logreg_cv.best_estimator_\n",
    "\n",
    "X_test = []\n",
    "manip_test = map(lambda s: int('_manip' in s), KaggleCameraDataset('../data/', train=False).X)\n",
    "manip_test = np.array(manip_test, dtype=np.float).reshape((-1, 1))\n",
    "for d in dirs:\n",
    "    X_b = inv_softmax(get_proba('../submissions/{0}/proba.csv'.format(d)))\n",
    "    X_test.append(X_b)\n",
    "X_test = [manip_test] + X_test\n",
    "X_test = np.hstack(X_test)\n",
    "P_test = clf.predict_proba(X_test)\n",
    "\n",
    "proba_to_subm(P_test, '../submissions/#237-submission.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## average all approaches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "proba_dirs = [\n",
    "    #\n",
    "    ##\n",
    "    ### generations 1, 2\n",
    "    (1., '0.913-#109-recomp'),   \n",
    "]\n",
    "\n",
    "weights, dirs = zip(*proba_dirs)\n",
    "P = avg_proba(*map(lambda p: '../submissions/{0}/proba.csv'.format(p), dirs), \n",
    "              weights=list(weights))\n",
    "proba_to_subm(P, '../submissions/all_averaged.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pseudo-labels [3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[198, 126, 237, 232, 229, 236, 249, 209, 214, 228]\n"
     ]
    }
   ],
   "source": [
    "P = get_proba('../submissions/0.977-#273/proba.csv')\n",
    "THRESHOLD = 0.999\n",
    "pseudo_ind = {}\n",
    "for c in xrange(10):\n",
    "    c_ind = np.arange(len(P))[P[:,c] > THRESHOLD].tolist()\n",
    "    c_ind.sort(key=lambda i: -P[i, c]) # sorted such that c_ind[0] is the most confident image\n",
    "    pseudo_ind[c] = c_ind\n",
    "PSEUDO_CLASSES = map(len, pseudo_ind.values())\n",
    "print PSEUDO_CLASSES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[20, 13, 24, 23, 23, 24, 25, 21, 21, 23]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PSEUDO_BLOCK_SIZE = 10\n",
    "N_PSEUDO_BLOCKS = [int(np.round(t/float(PSEUDO_BLOCK_SIZE))) for t in PSEUDO_CLASSES]\n",
    "N_PSEUDO_BLOCKS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_dataset = KaggleCameraDataset('../data/', train=False)\n",
    "for c in xrange(10):\n",
    "    c_ind = pseudo_ind[c]\n",
    "    RNG(seed=8888 + c).shuffle(c_ind)\n",
    "    class_blocks = []\n",
    "    class_manip = []\n",
    "    for _ in xrange(N_PSEUDO_BLOCKS[c]):\n",
    "        class_blocks.append([])\n",
    "        class_manip.append([])\n",
    "    pos = 0\n",
    "    for i in progress_iter(c_ind, True):\n",
    "        (x, m), _ = test_dataset[i]\n",
    "        x = np.asarray(x, dtype=np.uint8)\n",
    "        class_blocks[pos % N_PSEUDO_BLOCKS[c]].append(x)\n",
    "        class_manip[pos % N_PSEUDO_BLOCKS[c]].append(m)\n",
    "        pos += 1\n",
    "    for b in xrange(N_PSEUDO_BLOCKS[c]):\n",
    "        np.save('../data/X_pseudo_{0}_{1}.npy'.format(c, b), np.asarray(class_blocks[b], dtype=np.uint8))\n",
    "        np.save('../data/manip_pseudo_{0}_{1}.npy'.format(c, b), np.asarray(class_manip[b], dtype=np.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "N_PSEUDO_IMAGES_PER_BLOCK = []\n",
    "for c in xrange(10):\n",
    "    print c\n",
    "    N_PSEUDO_IMAGES_PER_BLOCK.append([])\n",
    "    for b in xrange(N_PSEUDO_BLOCKS[c]):\n",
    "        N_PSEUDO_IMAGES_PER_BLOCK[c].append(len(np.load('../data/X_pseudo_{0}_{1}.npy'.format(c, b))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2158\n",
      "[ 974.]\n"
     ]
    }
   ],
   "source": [
    "manip = []\n",
    "for c in xrange(10):\n",
    "    c_ind = pseudo_ind[c]\n",
    "    for i in c_ind:\n",
    "        (_, m), _ = test_dataset[i]\n",
    "        manip.append(m)\n",
    "print len(manip)\n",
    "print sum(manip)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pseudo-labels[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[248, 103, 237, 242, 236, 252, 251, 206, 223, 229]\n"
     ]
    }
   ],
   "source": [
    "P = get_proba('../submissions/0.960-d-#181/proba.csv')\n",
    "THRESHOLD = 0.99\n",
    "pseudo_ind = {}\n",
    "for c in xrange(10):\n",
    "    c_ind = np.arange(len(P))[P[:,c] > THRESHOLD].tolist()\n",
    "    c_ind.sort(key=lambda i: -P[i, c]) # sorted such that c_ind[0] is the most confident image\n",
    "    pseudo_ind[c] = c_ind\n",
    "print map(len, pseudo_ind.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1742, 361, 1386, 963, 2436]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pseudo_ind[0][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2138, 2171, 2220]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pseudo_ind[7][:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('../data/pseudo_ind.json', 'w') as f:\n",
    "    json.dump(pseudo_ind, f, indent=4, sort_keys=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('../data/pseudo_ind.json') as f:\n",
    "    pseudo_ind = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1742, 361, 1386, 963, 2436]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pseudo_ind['0'][:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load and save most confident images for validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_dataset = KaggleCameraDataset('../data/', train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N_PSEUDO_VAL_PER_CLASS = 24\n",
    "N_PSEUDO_VAL = N_PSEUDO_VAL_PER_CLASS * 10\n",
    "X_pseudo_val = np.zeros((N_PSEUDO_VAL, 512, 512, 3), dtype=np.uint8)\n",
    "\n",
    "pos = 0\n",
    "y_pseudo_val = []\n",
    "manip = []\n",
    "for c in xrange(10):\n",
    "    for i in pseudo_ind[str(c)][:N_PSEUDO_VAL_PER_CLASS]:\n",
    "        (x, m), _ = test_dataset[i]\n",
    "        X_pseudo_val[pos] = np.asarray(x, dtype=np.uint8)\n",
    "        pos += 1\n",
    "        manip.append(m)\n",
    "        y_pseudo_val.append(c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merge with existing validation, shuffle and save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_val_val = np.load('../data/X_val_val.npy')\n",
    "y_val_val = np.load('../data/y_val_val.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(480, 512, 512, 3) uint8\n"
     ]
    }
   ],
   "source": [
    "X_val_with_pseudo = np.concatenate((X_val_val, X_pseudo_val))\n",
    "print X_val_with_pseudo.shape, X_val_with_pseudo.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(480,) int64\n"
     ]
    }
   ],
   "source": [
    "y_val_with_pseudo = np.concatenate((y_val_val, np.asarray(y_pseudo_val)))\n",
    "print y_val_with_pseudo.shape, y_val_with_pseudo.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(480, 1) float32\n"
     ]
    }
   ],
   "source": [
    "manip_with_pseudo = np.concatenate((np.asarray([float32(0.)]*len(y_val_val)), manip))\n",
    "print manip_with_pseudo.shape, manip_with_pseudo.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ind = np.arange(480)\n",
    "RNG(seed=1234).shuffle(ind)\n",
    "X_val_with_pseudo = X_val_with_pseudo[ind]\n",
    "y_val_with_pseudo = y_val_with_pseudo[ind]\n",
    "manip_with_pseudo = manip_with_pseudo[ind]\n",
    "np.save('../data/X_val_with_pseudo.npy', X_val_with_pseudo)\n",
    "np.save('../data/y_val_with_pseudo.npy', y_val_with_pseudo)\n",
    "np.save('../data/manip_with_pseudo.npy', manip_with_pseudo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 68.], dtype=float32)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(manip_with_pseudo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "float32(0.)[0] < 0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### remaining pseudo-labels for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[224, 79, 213, 218, 212, 228, 227, 182, 199, 205]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for c in xrange(10):\n",
    "    pseudo_ind[c] = pseudo_ind[str(c)][N_PSEUDO_VAL_PER_CLASS:]\n",
    "    del pseudo_ind[str(c)]\n",
    "PSEUDO_CLASSES = map(len, pseudo_ind.values())\n",
    "PSEUDO_CLASSES"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### split into blocks of 8 images and save like regular train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[28, 10, 27, 27, 26, 28, 28, 23, 25, 26]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PSEUDO_BLOCK_SIZE = 8\n",
    "N_PSEUDO_BLOCKS = [int(np.round(t/float(PSEUDO_BLOCK_SIZE))) for t in PSEUDO_CLASSES]\n",
    "N_PSEUDO_BLOCKS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for c in xrange(10):\n",
    "    c_ind = pseudo_ind[c]\n",
    "    RNG(seed=8888 + c).shuffle(c_ind)\n",
    "    class_blocks = []\n",
    "    class_manip = []\n",
    "    for _ in xrange(N_PSEUDO_BLOCKS[c]):\n",
    "        class_blocks.append([])\n",
    "        class_manip.append([])\n",
    "    pos = 0\n",
    "    for i in progress_iter(c_ind, True):\n",
    "        (x, m), _ = test_dataset[i]\n",
    "        x = np.asarray(x, dtype=np.uint8)\n",
    "        class_blocks[pos % N_PSEUDO_BLOCKS[c]].append(x)\n",
    "        class_manip[pos % N_PSEUDO_BLOCKS[c]].append(m)\n",
    "        pos += 1\n",
    "    for b in xrange(N_PSEUDO_BLOCKS[c]):\n",
    "        np.save('../data/X_pseudo_{0}_{1}.npy'.format(c, b), np.asarray(class_blocks[b], dtype=np.uint8))\n",
    "        np.save('../data/manip_pseudo_{0}_{1}.npy'.format(c, b), np.asarray(class_manip[b], dtype=np.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### manip ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1987\n",
      "[ 972.]\n"
     ]
    }
   ],
   "source": [
    "manip = []\n",
    "for c in xrange(10):\n",
    "    c_ind = pseudo_ind[c]\n",
    "    for i in c_ind:\n",
    "        (_, m), _ = test_dataset[i]\n",
    "        manip.append(m)\n",
    "print len(manip)\n",
    "print sum(manip)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# pseudo-labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "835\n"
     ]
    }
   ],
   "source": [
    "# P from 0.908-#109\n",
    "ind = np.arange(2640)[P.max(axis=1) > 0.8]\n",
    "print len(ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data = KaggleCameraDataset('../data/', train=False)\n",
    "X = np.zeros((len(ind), 512, 512, 3), dtype=np.uint8)\n",
    "pos = 0\n",
    "for i in ind:\n",
    "    x = Image.open(test_data.X[i])\n",
    "    x = np.array(x, dtype=np.uint8)\n",
    "    X[pos, ...] = x\n",
    "    pos += 1\n",
    "y = np.argmax(P, axis=1)[ind]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### stratifically split into ~335 for validation and 500 for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(500,)\n"
     ]
    }
   ],
   "source": [
    "sss = StratifiedShuffleSplit(n_splits=1, train_size=500, random_state=1337)\n",
    "train_ind, val_ind = list(sss.split(X, y))[0]\n",
    "print train_ind.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('../data/X_pseudo_train.npy', X[train_ind])\n",
    "np.save('../data/y_pseudo_train.npy', y[train_ind])\n",
    "np.save('../data/X_pseudo_val.npy', X[val_ind])\n",
    "np.save('../data/y_pseudo_val.npy', y[val_ind])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
